from urllib import parse
import re
import json

import scrapy
from scrapy.http import Request
from CnblogsSpider.items import CnblogsspiderItem
from CnblogsSpider.items import CnblogsItemLoader
# import requests

from CnblogsSpider.utils import common


class CnblogsSpider(scrapy.Spider):
    name = 'cnblogs'
    allowed_domains = ['news.cnblogs.com']
    start_urls = ['http://news.cnblogs.com/']

    def parse(self, response):
        '''

        '''
        post_nodes = response.css('#news_list .news_block')[1:8]
        for post_node in post_nodes:
            image_url = post_node.css('.entry_summary a img::attr(src)').extract_first('')
            match_re = re.match('^http', image_url)
            if not match_re:
                image_url = 'https:' + image_url
            post_url = post_node.css('h2 a::attr(href)').extract_first('')
            yield Request(url=parse.urljoin(response.url, post_url), meta={'front_image_url': image_url}, callback=self.parse_detail)
        # 提取下一页交给scrapy进行下载
        # nex_url = response.xpath("//a[contains(text(), 'Next >')/@href]").extract_first('')
        # yield Request(url=parse.urljoin(response.url, nex_url), callback=self.parse)

    def parse_detail(self, response):
        match_re = re.match('.*?(\d+)', response.url)
        if match_re:
            # article_item = CnblogsspiderItem()
            # title = response.css('#news_title a::text').extract_first('')
            # create_date = response.css('#news_info span.time::text').extract_first('')
            # match_re1 = re.match('.*?(\d+.*)', create_date)
            # if match_re1:
            #     create_date = match_re1.group(1)
            # content = response.css('#news_content').extract_first('')
            # tag_list = response.css('.news_tags a::text').extract()
            # tags = ','.join(tag_list)
            #
            post_id = match_re.group(1)
            # # res = requests.get(parse.urljoin(response.url, '/NewsAjax/GetAjaxNewsInfo?contentId={}'.format(post_id)))
            # # j_data = json.loads(res.text)
            # article_item['title'] = title
            # article_item['create_date'] = create_date
            # article_item['content'] = content
            # article_item['tags'] = tags
            # article_item['front_image_url'] = [response.meta.get('front_image_url', '')]
            # article_item['title'] = title
            # article_item['url'] = response.url
            #
            item_loader = CnblogsItemLoader(item=CnblogsspiderItem(), response=response)
            item_loader.add_css('title', '#news_title a::text')
            item_loader.add_css('create_date', '#news_info span.time::text')
            item_loader.add_css('content', '#news_content')
            item_loader.add_css('tags', '.news_tags a::text')
            item_loader.add_value('url', response.url)
            item_loader.add_value('front_image_url', response.meta.get('front_image_url', ''))
            item_loader.add_css('title', '#news_title a::text')
            item_loader.add_css('title', '#news_title a::text')
            item_loader.add_css('title', '#news_title a::text')

            # article_item = item_loader.load_item()
            yield Request(url=parse.urljoin(response.url, '/NewsAjax/GetAjaxNewsInfo?contentId={}'.format(post_id)),
                          meta={'article_item': item_loader, 'url': response.url}, callback=self.parse_nums)

            # praise_nums = j_data['DiggCount']
            # fav_nums = j_data['TotalView']
            # comment_nums = j_data['CommentCount']

    def parse_nums(self, response):
        j_data = json.loads(response.text)
        item_loader = response.meta.get('article_item', '')
        praise_nums = j_data['DiggCount']
        fav_nums = j_data['TotalView']
        comment_nums = j_data['CommentCount']

        # article_item['praise_nums'] = praise_nums
        # article_item['fav_nums'] = fav_nums
        # article_item['comment_nums'] = comment_nums
        # article_item['url_object_id'] = common.get_md5(article_item['url'])

        item_loader.add_value('praise_nums', praise_nums)
        item_loader.add_value('fav_nums', fav_nums)
        item_loader.add_value('comment_nums', comment_nums)
        item_loader.add_value('url_object_id', common.get_md5(response.meta.get('url', '')))

        article_item = item_loader.load_item()

        yield article_item

